from urllib import parse, request
import json
from pymongo import MongoClient

def get_caller(input_url, collection, func):
    func(input_url, collection)

def get_method(input_url, collection):
    print(input_url)
    res_data = request.urlopen(input_url)
    res = res_data.read()
    res_string = str(res, encoding="utf-8")
    res_json = json.loads(res_string)
    res_result = res_json['result']
    if ('error' in res_result):
        with open('error.txt', 'a') as f:
            f.write(str(input_url) + '\n')
            f.write(str(res_result) + '\n')
            f.write('__________________________________________________\n')
    else:
        collection.insert_one(res_result)

    # print('********************' + str(res_json))

#data set
url = 'http://api.steampowered.com/IDOTA2Match_570/GetMatchDetails/v1'
start_match_ID_num = 3567996028
search_match_numbers = 28994197

#mongo connect
client = MongoClient('mongodb://root:19930309@dds-uf6773d77c87f4341.mongodb.rds.aliyuncs.com:3717,dds-uf6773d77c87f4342.mongodb.rds.aliyuncs.com:3717/admin?replicaSet=mgset-4868049')
db = client.dota2data
collection = db.dota2data

for i in range(search_match_numbers):
    param = {'match_id': str(start_match_ID_num+i), 'key': 'EFA1E81676FCC47157EA871A67741EF5'}
    param = parse.urlencode(param)
    url_counter = url + '?' + param
    get_caller(url_counter, collection, get_method)
    if(i%1000 is 0):
        print('当前已经处理到第 %s 个数据' % (str(i)))

